900260 Automated Visual Event Detection and Classification : AVED Beowulf RAID administrative utilities

This page last changed on May 28, 2013 by dcline.

Introduction

These are general administrative utilities for managing the MBARI Beowulf RAID.
They are described here in case this is helpful for someone else.

Utilities installed in /usr/local/bin

Install into crontab

#every 10 minutes, remount in case a mount point is intermittent
0-59/10 * * * * /bin/mount -a >/dev/null 2>&1
#on the first of every month, remove files older than 90 days
00 00 01 * * /usr/local/bin/cleanscratchfiles
#at midnight daily rebuild scratch in case new users added or permissions changed
59 23 * * * /usr/local/bin/buildscratch
#check disk space every hour and send root alert if getting too full
0 * * * * /usr/local/bin/diskalert

diskalert

#!/bin/sh
# set -x
# Shell script to monitor or watch the disk space
# It will send an email to $ADMIN, if the (free available) percentage of space is >= 90%.
# -------------------------------------------------------------------------
# Set admin email so that you can get email.
ADMIN="root"

# set alert level 90% is default
ALERT=90

# Exclude list of unwanted monitoring, if several partions then use "|" to separate the partitions.
# An example: EXCLUDE_LIST="/dev/hdd1|/dev/hdc5"

EXCLUDE_LIST="tempest:/users|tempest:/tempbox:tornado:/Engineering|tornado:/vol/vol1/ProjectLibrary|nanomia.private.net:/nanomiaRAID"
#
#::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
#
function main_prog() {
while read output;
do
  #echo $output
  usep=$(echo $output | awk '{ print $1}' | cut -d'%' -f1)
  partition=$(echo $output | awk '{print $2}')
  if [ $usep -ge $ALERT ] ; then
     echo "Running out of space \"$partition ($usep%)\" on server $(hostname), $(date)" |
     mail -s "ALERT: Almost out of disk space $usep% on $(hostname)" $ADMIN
  fi
done
}

if [ "$EXCLUDE_LIST" != "" ] ; then
  df -lH | grep -vE "^Filesystem|tmpfs|cdrom|${EXCLUDE_LIST}" | awk '{print $5 " " $6}' | main_prog
else
  df -lH | grep -vE "^Filesystem|tmpfs|cdrom" | awk '{print $5 " " $6}' | main_prog
fi

buildscratch

! /bin/bash
#set -x
USERID=`id -u`
if [ $USERID != 0 ]; then
        echo "You must be root to rue buildscratch"
        exit 0
fi

function dowork() {
echo "Building $1"

#remove all directories and recreate using file containing users
users=$(cat /etc/passwd |grep 20| cut -f1 -d ":")

pushd /mnt/scratch

#change permissions to include all users
chmod a+rwx /mnt/scratch

#change owner for new directories
for dir in $users
do
        if [ ! -d $dir ]; then
                echo "mkdir $dir"
                mkdir $dir
        fi
        echo "chown $dir:users $dir"
        chown $dir:users $dir
        echo "chmod 0777 $dir "
        chmod 777 $dir
done
chown aved:aved aved
popd

#refresh scratch space NFS mounts on all nodes
#cexec '/bin/umount /mnt/scratch;/bin/mount /mnt/scratch'
}

for s in "/mnt/scratch" "/mnt/scratch2"
do
        dowork $s
done

#rebuild video directories for capture
echo "Rebuilding video capture directories"
if [ ! -d /mnt/scratch/video ]; then
        mkdir -p /mnt/scratch/video/capture
fi
chmod -Rf a+rwx /mnt/scratch/video

#rebuild directories for condor
if [ ! -d /mnt/scratch/condor ]; then
        mkdir -p /mnt/scratch/condor -m=0755
fi
chown daemon:root /mnt/scratch/condor

RAID format

Install xfsprogs

yum install xfsprogs

Reformat NSTOR RAID

fdisk /dev/sda
pvcreate /dev/sda
vgdisplay
lvcreate -i2 -I4 -l 2198.7G -namedev AVEDRAID

Reformat RAID filesystem to XFS

mkfs.xfs -f - L "AVEDRAID" /dev/sda